{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\sklearn\\cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n",
      "C:\\Anaconda3\\envs\\tensorflow\\lib\\site-packages\\sklearn\\grid_search.py:42: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. This module will be removed in 0.20.\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import scipy as sp\n",
    "import pandas as pd\n",
    "import sklearn as skl\n",
    "import matplotlib.pyplot as plt\n",
    "import xgboost as xgb\n",
    "import pickle\n",
    "from xgboost.sklearn import XGBClassifier\n",
    "from sklearn import cross_validation, metrics   #Additional scklearn functions\n",
    "from sklearn.grid_search import GridSearchCV   #Perforing grid search\n",
    "from sklearn.cross_validation import train_test_split, cross_val_score, KFold\n",
    "from sklearn.metrics import log_loss, confusion_matrix, accuracy_score\n",
    "from sklearn.preprocessing import minmax_scale\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load training data\n",
    "max_features = 1024\n",
    "train = pd.read_csv('data/train.csv', header=None, sep=',', usecols = list(range(0,max_features)))\n",
    "label = pd.read_csv(\"data/train_label.csv\", sep=',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = train\n",
    "number_of_nan = X.isnull().sum(axis=1)\n",
    "X = X.fillna(0)\n",
    "X = X.astype(int)\n",
    "number_valid = max_features - number_of_nan\n",
    "X_describe = X.apply(pd.DataFrame.describe, axis=1)\n",
    "X = pd.concat([X, number_of_nan, number_valid, X_describe], axis=1)\n",
    "X.columns.values[1024] = \"number_of_nan\"\n",
    "X.columns.values[1025] = \"number_valid\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = label.iloc[:, -1]\n",
    "\n",
    "x_train, x_test, y_train, y_test = train_test_split(X, y, test_size=0.05, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-logloss:0.626909\n",
      "Will train until validation_0-logloss hasn't improved in 100 rounds.\n",
      "[1]\tvalidation_0-logloss:0.57162\n",
      "[2]\tvalidation_0-logloss:0.525737\n",
      "[3]\tvalidation_0-logloss:0.48707\n",
      "[4]\tvalidation_0-logloss:0.453427\n",
      "[5]\tvalidation_0-logloss:0.420634\n",
      "[6]\tvalidation_0-logloss:0.38918\n",
      "[7]\tvalidation_0-logloss:0.366433\n",
      "[8]\tvalidation_0-logloss:0.341959\n",
      "[9]\tvalidation_0-logloss:0.319798\n",
      "[10]\tvalidation_0-logloss:0.300399\n",
      "[11]\tvalidation_0-logloss:0.284087\n",
      "[12]\tvalidation_0-logloss:0.269005\n",
      "[13]\tvalidation_0-logloss:0.255484\n",
      "[14]\tvalidation_0-logloss:0.243773\n",
      "[15]\tvalidation_0-logloss:0.232777\n",
      "[16]\tvalidation_0-logloss:0.223419\n",
      "[17]\tvalidation_0-logloss:0.214943\n",
      "[18]\tvalidation_0-logloss:0.207696\n",
      "[19]\tvalidation_0-logloss:0.199876\n",
      "[20]\tvalidation_0-logloss:0.19351\n",
      "[21]\tvalidation_0-logloss:0.18786\n",
      "[22]\tvalidation_0-logloss:0.182258\n",
      "[23]\tvalidation_0-logloss:0.178127\n",
      "[24]\tvalidation_0-logloss:0.172549\n",
      "[25]\tvalidation_0-logloss:0.169023\n",
      "[26]\tvalidation_0-logloss:0.164423\n",
      "[27]\tvalidation_0-logloss:0.160208\n",
      "[28]\tvalidation_0-logloss:0.157057\n",
      "[29]\tvalidation_0-logloss:0.153332\n",
      "[30]\tvalidation_0-logloss:0.150322\n",
      "[31]\tvalidation_0-logloss:0.148037\n",
      "[32]\tvalidation_0-logloss:0.146284\n",
      "[33]\tvalidation_0-logloss:0.142861\n",
      "[34]\tvalidation_0-logloss:0.141188\n",
      "[35]\tvalidation_0-logloss:0.138363\n",
      "[36]\tvalidation_0-logloss:0.136485\n",
      "[37]\tvalidation_0-logloss:0.134169\n",
      "[38]\tvalidation_0-logloss:0.133417\n",
      "[39]\tvalidation_0-logloss:0.13082\n",
      "[40]\tvalidation_0-logloss:0.129569\n",
      "[41]\tvalidation_0-logloss:0.128296\n",
      "[42]\tvalidation_0-logloss:0.126491\n",
      "[43]\tvalidation_0-logloss:0.125229\n",
      "[44]\tvalidation_0-logloss:0.122846\n",
      "[45]\tvalidation_0-logloss:0.121522\n",
      "[46]\tvalidation_0-logloss:0.121007\n",
      "[47]\tvalidation_0-logloss:0.120182\n",
      "[48]\tvalidation_0-logloss:0.119535\n",
      "[49]\tvalidation_0-logloss:0.118456\n",
      "[50]\tvalidation_0-logloss:0.117228\n",
      "[51]\tvalidation_0-logloss:0.115873\n",
      "[52]\tvalidation_0-logloss:0.114542\n",
      "[53]\tvalidation_0-logloss:0.114219\n",
      "[54]\tvalidation_0-logloss:0.113627\n",
      "[55]\tvalidation_0-logloss:0.112969\n",
      "[56]\tvalidation_0-logloss:0.112343\n",
      "[57]\tvalidation_0-logloss:0.111842\n",
      "[58]\tvalidation_0-logloss:0.111364\n",
      "[59]\tvalidation_0-logloss:0.110819\n",
      "[60]\tvalidation_0-logloss:0.110583\n",
      "[61]\tvalidation_0-logloss:0.110044\n",
      "[62]\tvalidation_0-logloss:0.109248\n",
      "[63]\tvalidation_0-logloss:0.108585\n",
      "[64]\tvalidation_0-logloss:0.107083\n",
      "[65]\tvalidation_0-logloss:0.106516\n",
      "[66]\tvalidation_0-logloss:0.106264\n",
      "[67]\tvalidation_0-logloss:0.105676\n",
      "[68]\tvalidation_0-logloss:0.104934\n",
      "[69]\tvalidation_0-logloss:0.104234\n",
      "[70]\tvalidation_0-logloss:0.103694\n",
      "[71]\tvalidation_0-logloss:0.103444\n",
      "[72]\tvalidation_0-logloss:0.10324\n",
      "[73]\tvalidation_0-logloss:0.102519\n",
      "[74]\tvalidation_0-logloss:0.10193\n",
      "[75]\tvalidation_0-logloss:0.101387\n",
      "[76]\tvalidation_0-logloss:0.100915\n",
      "[77]\tvalidation_0-logloss:0.100264\n",
      "[78]\tvalidation_0-logloss:0.099704\n",
      "[79]\tvalidation_0-logloss:0.099336\n",
      "[80]\tvalidation_0-logloss:0.098777\n",
      "[81]\tvalidation_0-logloss:0.098566\n",
      "[82]\tvalidation_0-logloss:0.098025\n",
      "[83]\tvalidation_0-logloss:0.097759\n",
      "[84]\tvalidation_0-logloss:0.097475\n",
      "[85]\tvalidation_0-logloss:0.097084\n",
      "[86]\tvalidation_0-logloss:0.096921\n",
      "[87]\tvalidation_0-logloss:0.096273\n",
      "[88]\tvalidation_0-logloss:0.095944\n",
      "[89]\tvalidation_0-logloss:0.095366\n",
      "[90]\tvalidation_0-logloss:0.094919\n",
      "[91]\tvalidation_0-logloss:0.094458\n",
      "[92]\tvalidation_0-logloss:0.094183\n",
      "[93]\tvalidation_0-logloss:0.09397\n",
      "[94]\tvalidation_0-logloss:0.0938\n",
      "[95]\tvalidation_0-logloss:0.093517\n",
      "[96]\tvalidation_0-logloss:0.093273\n",
      "[97]\tvalidation_0-logloss:0.092721\n",
      "[98]\tvalidation_0-logloss:0.092356\n",
      "[99]\tvalidation_0-logloss:0.092208\n",
      "[100]\tvalidation_0-logloss:0.091319\n",
      "[101]\tvalidation_0-logloss:0.090723\n",
      "[102]\tvalidation_0-logloss:0.090549\n",
      "[103]\tvalidation_0-logloss:0.090204\n",
      "[104]\tvalidation_0-logloss:0.089576\n",
      "[105]\tvalidation_0-logloss:0.0893\n",
      "[106]\tvalidation_0-logloss:0.089084\n",
      "[107]\tvalidation_0-logloss:0.088869\n",
      "[108]\tvalidation_0-logloss:0.088649\n",
      "[109]\tvalidation_0-logloss:0.088366\n",
      "[110]\tvalidation_0-logloss:0.088174\n",
      "[111]\tvalidation_0-logloss:0.087802\n",
      "[112]\tvalidation_0-logloss:0.087701\n",
      "[113]\tvalidation_0-logloss:0.087099\n",
      "[114]\tvalidation_0-logloss:0.086976\n",
      "[115]\tvalidation_0-logloss:0.086692\n",
      "[116]\tvalidation_0-logloss:0.086579\n",
      "[117]\tvalidation_0-logloss:0.086162\n",
      "[118]\tvalidation_0-logloss:0.085545\n",
      "[119]\tvalidation_0-logloss:0.085389\n",
      "[120]\tvalidation_0-logloss:0.085294\n",
      "[121]\tvalidation_0-logloss:0.084999\n",
      "[122]\tvalidation_0-logloss:0.084798\n",
      "[123]\tvalidation_0-logloss:0.084646\n",
      "[124]\tvalidation_0-logloss:0.084357\n",
      "[125]\tvalidation_0-logloss:0.084219\n",
      "[126]\tvalidation_0-logloss:0.08373\n",
      "[127]\tvalidation_0-logloss:0.083576\n",
      "[128]\tvalidation_0-logloss:0.083408\n",
      "[129]\tvalidation_0-logloss:0.083109\n",
      "[130]\tvalidation_0-logloss:0.082948\n",
      "[131]\tvalidation_0-logloss:0.082758\n",
      "[132]\tvalidation_0-logloss:0.082707\n",
      "[133]\tvalidation_0-logloss:0.082609\n",
      "[134]\tvalidation_0-logloss:0.082327\n",
      "[135]\tvalidation_0-logloss:0.082246\n",
      "[136]\tvalidation_0-logloss:0.081988\n",
      "[137]\tvalidation_0-logloss:0.081909\n",
      "[138]\tvalidation_0-logloss:0.08177\n",
      "[139]\tvalidation_0-logloss:0.08143\n",
      "[140]\tvalidation_0-logloss:0.08134\n",
      "[141]\tvalidation_0-logloss:0.0812\n",
      "[142]\tvalidation_0-logloss:0.080894\n",
      "[143]\tvalidation_0-logloss:0.080762\n",
      "[144]\tvalidation_0-logloss:0.080602\n",
      "[145]\tvalidation_0-logloss:0.080204\n",
      "[146]\tvalidation_0-logloss:0.080107\n",
      "[147]\tvalidation_0-logloss:0.079836\n",
      "[148]\tvalidation_0-logloss:0.079552\n",
      "[149]\tvalidation_0-logloss:0.079254\n",
      "[150]\tvalidation_0-logloss:0.079011\n",
      "[151]\tvalidation_0-logloss:0.078915\n",
      "[152]\tvalidation_0-logloss:0.078813\n",
      "[153]\tvalidation_0-logloss:0.078567\n",
      "[154]\tvalidation_0-logloss:0.078149\n",
      "[155]\tvalidation_0-logloss:0.077971\n",
      "[156]\tvalidation_0-logloss:0.077753\n",
      "[157]\tvalidation_0-logloss:0.077559\n",
      "[158]\tvalidation_0-logloss:0.077491\n",
      "[159]\tvalidation_0-logloss:0.07737\n",
      "[160]\tvalidation_0-logloss:0.077209\n",
      "[161]\tvalidation_0-logloss:0.077105\n",
      "[162]\tvalidation_0-logloss:0.076911\n",
      "[163]\tvalidation_0-logloss:0.076653\n",
      "[164]\tvalidation_0-logloss:0.076559\n",
      "[165]\tvalidation_0-logloss:0.076492\n",
      "[166]\tvalidation_0-logloss:0.076412\n",
      "[167]\tvalidation_0-logloss:0.07639\n",
      "[168]\tvalidation_0-logloss:0.076255\n",
      "[169]\tvalidation_0-logloss:0.076203\n",
      "[170]\tvalidation_0-logloss:0.075618\n",
      "[171]\tvalidation_0-logloss:0.075536\n",
      "[172]\tvalidation_0-logloss:0.075454\n",
      "[173]\tvalidation_0-logloss:0.075254\n",
      "[174]\tvalidation_0-logloss:0.075175\n",
      "[175]\tvalidation_0-logloss:0.07506\n",
      "[176]\tvalidation_0-logloss:0.07479\n",
      "[177]\tvalidation_0-logloss:0.074626\n",
      "[178]\tvalidation_0-logloss:0.074471\n",
      "[179]\tvalidation_0-logloss:0.074295\n",
      "[180]\tvalidation_0-logloss:0.074145\n",
      "[181]\tvalidation_0-logloss:0.074031\n",
      "[182]\tvalidation_0-logloss:0.073958\n",
      "[183]\tvalidation_0-logloss:0.073779\n",
      "[184]\tvalidation_0-logloss:0.073502\n",
      "[185]\tvalidation_0-logloss:0.073446\n",
      "[186]\tvalidation_0-logloss:0.07339\n",
      "[187]\tvalidation_0-logloss:0.073218\n",
      "[188]\tvalidation_0-logloss:0.073104\n",
      "[189]\tvalidation_0-logloss:0.073015\n",
      "[190]\tvalidation_0-logloss:0.072926\n",
      "[191]\tvalidation_0-logloss:0.07272\n",
      "[192]\tvalidation_0-logloss:0.072653\n",
      "[193]\tvalidation_0-logloss:0.072603\n",
      "[194]\tvalidation_0-logloss:0.072503\n",
      "[195]\tvalidation_0-logloss:0.072496\n",
      "[196]\tvalidation_0-logloss:0.072426\n",
      "[197]\tvalidation_0-logloss:0.072331\n",
      "[198]\tvalidation_0-logloss:0.072239\n",
      "[199]\tvalidation_0-logloss:0.072045\n",
      "[200]\tvalidation_0-logloss:0.071734\n",
      "[201]\tvalidation_0-logloss:0.071688\n",
      "[202]\tvalidation_0-logloss:0.071265\n",
      "[203]\tvalidation_0-logloss:0.071092\n",
      "[204]\tvalidation_0-logloss:0.070932\n",
      "[205]\tvalidation_0-logloss:0.070891\n",
      "[206]\tvalidation_0-logloss:0.070739\n",
      "[207]\tvalidation_0-logloss:0.070629\n",
      "[208]\tvalidation_0-logloss:0.070606\n",
      "[209]\tvalidation_0-logloss:0.070438\n",
      "[210]\tvalidation_0-logloss:0.070382\n",
      "[211]\tvalidation_0-logloss:0.070254\n",
      "[212]\tvalidation_0-logloss:0.070041\n",
      "[213]\tvalidation_0-logloss:0.070005\n",
      "[214]\tvalidation_0-logloss:0.06994\n",
      "[215]\tvalidation_0-logloss:0.069814\n",
      "[216]\tvalidation_0-logloss:0.069725\n",
      "[217]\tvalidation_0-logloss:0.069636\n",
      "[218]\tvalidation_0-logloss:0.069139\n",
      "[219]\tvalidation_0-logloss:0.069084\n",
      "[220]\tvalidation_0-logloss:0.069071\n",
      "[221]\tvalidation_0-logloss:0.068929\n",
      "[222]\tvalidation_0-logloss:0.068733\n",
      "[223]\tvalidation_0-logloss:0.068428\n",
      "[224]\tvalidation_0-logloss:0.068384\n",
      "[225]\tvalidation_0-logloss:0.06827\n",
      "[226]\tvalidation_0-logloss:0.068074\n",
      "[227]\tvalidation_0-logloss:0.067944\n",
      "[228]\tvalidation_0-logloss:0.067869\n",
      "[229]\tvalidation_0-logloss:0.067548\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[230]\tvalidation_0-logloss:0.06753\n",
      "[231]\tvalidation_0-logloss:0.067359\n",
      "[232]\tvalidation_0-logloss:0.067182\n",
      "[233]\tvalidation_0-logloss:0.067147\n",
      "[234]\tvalidation_0-logloss:0.06716\n",
      "[235]\tvalidation_0-logloss:0.066998\n",
      "[236]\tvalidation_0-logloss:0.066893\n",
      "[237]\tvalidation_0-logloss:0.066788\n",
      "[238]\tvalidation_0-logloss:0.066722\n",
      "[239]\tvalidation_0-logloss:0.066689\n",
      "[240]\tvalidation_0-logloss:0.066626\n",
      "[241]\tvalidation_0-logloss:0.066592\n",
      "[242]\tvalidation_0-logloss:0.066564\n",
      "[243]\tvalidation_0-logloss:0.066558\n",
      "[244]\tvalidation_0-logloss:0.066463\n",
      "[245]\tvalidation_0-logloss:0.066388\n",
      "[246]\tvalidation_0-logloss:0.066326\n",
      "[247]\tvalidation_0-logloss:0.066209\n",
      "[248]\tvalidation_0-logloss:0.066204\n",
      "[249]\tvalidation_0-logloss:0.066175\n",
      "[250]\tvalidation_0-logloss:0.065933\n",
      "[251]\tvalidation_0-logloss:0.065877\n",
      "[252]\tvalidation_0-logloss:0.065697\n",
      "[253]\tvalidation_0-logloss:0.065752\n",
      "[254]\tvalidation_0-logloss:0.065682\n",
      "[255]\tvalidation_0-logloss:0.065387\n",
      "[256]\tvalidation_0-logloss:0.065342\n",
      "[257]\tvalidation_0-logloss:0.065192\n",
      "[258]\tvalidation_0-logloss:0.065093\n",
      "[259]\tvalidation_0-logloss:0.065064\n",
      "[260]\tvalidation_0-logloss:0.065059\n",
      "[261]\tvalidation_0-logloss:0.064936\n",
      "[262]\tvalidation_0-logloss:0.064889\n",
      "[263]\tvalidation_0-logloss:0.064754\n",
      "[264]\tvalidation_0-logloss:0.064707\n",
      "[265]\tvalidation_0-logloss:0.064628\n",
      "[266]\tvalidation_0-logloss:0.064576\n",
      "[267]\tvalidation_0-logloss:0.064469\n",
      "[268]\tvalidation_0-logloss:0.064328\n",
      "[269]\tvalidation_0-logloss:0.064258\n",
      "[270]\tvalidation_0-logloss:0.064231\n",
      "[271]\tvalidation_0-logloss:0.064192\n",
      "[272]\tvalidation_0-logloss:0.064082\n",
      "[273]\tvalidation_0-logloss:0.064025\n",
      "[274]\tvalidation_0-logloss:0.06396\n",
      "[275]\tvalidation_0-logloss:0.063882\n",
      "[276]\tvalidation_0-logloss:0.063832\n",
      "[277]\tvalidation_0-logloss:0.063653\n",
      "[278]\tvalidation_0-logloss:0.063626\n",
      "[279]\tvalidation_0-logloss:0.063563\n",
      "[280]\tvalidation_0-logloss:0.063564\n",
      "[281]\tvalidation_0-logloss:0.06356\n",
      "[282]\tvalidation_0-logloss:0.063569\n",
      "[283]\tvalidation_0-logloss:0.063502\n",
      "[284]\tvalidation_0-logloss:0.063461\n",
      "[285]\tvalidation_0-logloss:0.063389\n",
      "[286]\tvalidation_0-logloss:0.063267\n",
      "[287]\tvalidation_0-logloss:0.063228\n",
      "[288]\tvalidation_0-logloss:0.063203\n",
      "[289]\tvalidation_0-logloss:0.063172\n",
      "[290]\tvalidation_0-logloss:0.062977\n",
      "[291]\tvalidation_0-logloss:0.062965\n",
      "[292]\tvalidation_0-logloss:0.06293\n",
      "[293]\tvalidation_0-logloss:0.062649\n",
      "[294]\tvalidation_0-logloss:0.06267\n",
      "[295]\tvalidation_0-logloss:0.062568\n",
      "[296]\tvalidation_0-logloss:0.062618\n",
      "[297]\tvalidation_0-logloss:0.062587\n",
      "[298]\tvalidation_0-logloss:0.062579\n",
      "[299]\tvalidation_0-logloss:0.062494\n",
      "[300]\tvalidation_0-logloss:0.062472\n",
      "[301]\tvalidation_0-logloss:0.062379\n",
      "[302]\tvalidation_0-logloss:0.062396\n",
      "[303]\tvalidation_0-logloss:0.062376\n",
      "[304]\tvalidation_0-logloss:0.062323\n",
      "[305]\tvalidation_0-logloss:0.062236\n",
      "[306]\tvalidation_0-logloss:0.062087\n",
      "[307]\tvalidation_0-logloss:0.061985\n",
      "[308]\tvalidation_0-logloss:0.061952\n",
      "[309]\tvalidation_0-logloss:0.061919\n",
      "[310]\tvalidation_0-logloss:0.061824\n",
      "[311]\tvalidation_0-logloss:0.061827\n",
      "[312]\tvalidation_0-logloss:0.061794\n",
      "[313]\tvalidation_0-logloss:0.061821\n",
      "[314]\tvalidation_0-logloss:0.061851\n",
      "[315]\tvalidation_0-logloss:0.061847\n",
      "[316]\tvalidation_0-logloss:0.061819\n",
      "[317]\tvalidation_0-logloss:0.061775\n",
      "[318]\tvalidation_0-logloss:0.06177\n",
      "[319]\tvalidation_0-logloss:0.061706\n",
      "[320]\tvalidation_0-logloss:0.061602\n",
      "[321]\tvalidation_0-logloss:0.061497\n",
      "[322]\tvalidation_0-logloss:0.061491\n",
      "[323]\tvalidation_0-logloss:0.061202\n",
      "[324]\tvalidation_0-logloss:0.061167\n",
      "[325]\tvalidation_0-logloss:0.061062\n",
      "[326]\tvalidation_0-logloss:0.060971\n",
      "[327]\tvalidation_0-logloss:0.060884\n",
      "[328]\tvalidation_0-logloss:0.060858\n",
      "[329]\tvalidation_0-logloss:0.060867\n",
      "[330]\tvalidation_0-logloss:0.06088\n",
      "[331]\tvalidation_0-logloss:0.060745\n",
      "[332]\tvalidation_0-logloss:0.060744\n",
      "[333]\tvalidation_0-logloss:0.06074\n",
      "[334]\tvalidation_0-logloss:0.060675\n",
      "[335]\tvalidation_0-logloss:0.060613\n",
      "[336]\tvalidation_0-logloss:0.060609\n",
      "[337]\tvalidation_0-logloss:0.060481\n",
      "[338]\tvalidation_0-logloss:0.060404\n",
      "[339]\tvalidation_0-logloss:0.060353\n",
      "[340]\tvalidation_0-logloss:0.060324\n",
      "[341]\tvalidation_0-logloss:0.060308\n",
      "[342]\tvalidation_0-logloss:0.06026\n",
      "[343]\tvalidation_0-logloss:0.060187\n",
      "[344]\tvalidation_0-logloss:0.060157\n",
      "[345]\tvalidation_0-logloss:0.060112\n",
      "[346]\tvalidation_0-logloss:0.060046\n",
      "[347]\tvalidation_0-logloss:0.060001\n",
      "[348]\tvalidation_0-logloss:0.060003\n",
      "[349]\tvalidation_0-logloss:0.059945\n",
      "[350]\tvalidation_0-logloss:0.059927\n",
      "[351]\tvalidation_0-logloss:0.05989\n",
      "[352]\tvalidation_0-logloss:0.059858\n",
      "[353]\tvalidation_0-logloss:0.059866\n",
      "[354]\tvalidation_0-logloss:0.059774\n",
      "[355]\tvalidation_0-logloss:0.059743\n",
      "[356]\tvalidation_0-logloss:0.059712\n",
      "[357]\tvalidation_0-logloss:0.059705\n",
      "[358]\tvalidation_0-logloss:0.059678\n",
      "[359]\tvalidation_0-logloss:0.059641\n",
      "[360]\tvalidation_0-logloss:0.05959\n",
      "[361]\tvalidation_0-logloss:0.059451\n",
      "[362]\tvalidation_0-logloss:0.059391\n",
      "[363]\tvalidation_0-logloss:0.059371\n",
      "[364]\tvalidation_0-logloss:0.05932\n",
      "[365]\tvalidation_0-logloss:0.059149\n",
      "[366]\tvalidation_0-logloss:0.059151\n",
      "[367]\tvalidation_0-logloss:0.059109\n",
      "[368]\tvalidation_0-logloss:0.059119\n",
      "[369]\tvalidation_0-logloss:0.059001\n",
      "[370]\tvalidation_0-logloss:0.058877\n",
      "[371]\tvalidation_0-logloss:0.058901\n",
      "[372]\tvalidation_0-logloss:0.058783\n",
      "[373]\tvalidation_0-logloss:0.058728\n",
      "[374]\tvalidation_0-logloss:0.058703\n",
      "[375]\tvalidation_0-logloss:0.058611\n",
      "[376]\tvalidation_0-logloss:0.058607\n",
      "[377]\tvalidation_0-logloss:0.058491\n",
      "[378]\tvalidation_0-logloss:0.058484\n",
      "[379]\tvalidation_0-logloss:0.058407\n",
      "[380]\tvalidation_0-logloss:0.058405\n",
      "[381]\tvalidation_0-logloss:0.058309\n",
      "[382]\tvalidation_0-logloss:0.058192\n",
      "[383]\tvalidation_0-logloss:0.058155\n",
      "[384]\tvalidation_0-logloss:0.058168\n",
      "[385]\tvalidation_0-logloss:0.058101\n",
      "[386]\tvalidation_0-logloss:0.058047\n",
      "[387]\tvalidation_0-logloss:0.058077\n",
      "[388]\tvalidation_0-logloss:0.058044\n",
      "[389]\tvalidation_0-logloss:0.058045\n",
      "[390]\tvalidation_0-logloss:0.057997\n",
      "[391]\tvalidation_0-logloss:0.057831\n",
      "[392]\tvalidation_0-logloss:0.057797\n",
      "[393]\tvalidation_0-logloss:0.057776\n",
      "[394]\tvalidation_0-logloss:0.057758\n",
      "[395]\tvalidation_0-logloss:0.05776\n",
      "[396]\tvalidation_0-logloss:0.057781\n",
      "[397]\tvalidation_0-logloss:0.057814\n",
      "[398]\tvalidation_0-logloss:0.057795\n",
      "[399]\tvalidation_0-logloss:0.057724\n",
      "[400]\tvalidation_0-logloss:0.057652\n",
      "[401]\tvalidation_0-logloss:0.057617\n",
      "[402]\tvalidation_0-logloss:0.057598\n",
      "[403]\tvalidation_0-logloss:0.057608\n",
      "[404]\tvalidation_0-logloss:0.057612\n",
      "[405]\tvalidation_0-logloss:0.057502\n",
      "[406]\tvalidation_0-logloss:0.057479\n",
      "[407]\tvalidation_0-logloss:0.057433\n",
      "[408]\tvalidation_0-logloss:0.057438\n",
      "[409]\tvalidation_0-logloss:0.05742\n",
      "[410]\tvalidation_0-logloss:0.057441\n",
      "[411]\tvalidation_0-logloss:0.057443\n",
      "[412]\tvalidation_0-logloss:0.057318\n",
      "[413]\tvalidation_0-logloss:0.057395\n",
      "[414]\tvalidation_0-logloss:0.057373\n",
      "[415]\tvalidation_0-logloss:0.057194\n",
      "[416]\tvalidation_0-logloss:0.057078\n",
      "[417]\tvalidation_0-logloss:0.057057\n",
      "[418]\tvalidation_0-logloss:0.057004\n",
      "[419]\tvalidation_0-logloss:0.056998\n",
      "[420]\tvalidation_0-logloss:0.056998\n",
      "[421]\tvalidation_0-logloss:0.05701\n",
      "[422]\tvalidation_0-logloss:0.056911\n",
      "[423]\tvalidation_0-logloss:0.056884\n",
      "[424]\tvalidation_0-logloss:0.056866\n",
      "[425]\tvalidation_0-logloss:0.056854\n",
      "[426]\tvalidation_0-logloss:0.056816\n",
      "[427]\tvalidation_0-logloss:0.056672\n",
      "[428]\tvalidation_0-logloss:0.056572\n",
      "[429]\tvalidation_0-logloss:0.056574\n",
      "[430]\tvalidation_0-logloss:0.056533\n",
      "[431]\tvalidation_0-logloss:0.056487\n",
      "[432]\tvalidation_0-logloss:0.056439\n",
      "[433]\tvalidation_0-logloss:0.056455\n",
      "[434]\tvalidation_0-logloss:0.056411\n",
      "[435]\tvalidation_0-logloss:0.0564\n",
      "[436]\tvalidation_0-logloss:0.056396\n",
      "[437]\tvalidation_0-logloss:0.056395\n",
      "[438]\tvalidation_0-logloss:0.056389\n",
      "[439]\tvalidation_0-logloss:0.056365\n",
      "[440]\tvalidation_0-logloss:0.056325\n",
      "[441]\tvalidation_0-logloss:0.056327\n",
      "[442]\tvalidation_0-logloss:0.056336\n",
      "[443]\tvalidation_0-logloss:0.056328\n",
      "[444]\tvalidation_0-logloss:0.056246\n",
      "[445]\tvalidation_0-logloss:0.056237\n",
      "[446]\tvalidation_0-logloss:0.056151\n",
      "[447]\tvalidation_0-logloss:0.05614\n",
      "[448]\tvalidation_0-logloss:0.056104\n",
      "[449]\tvalidation_0-logloss:0.056051\n",
      "[450]\tvalidation_0-logloss:0.056009\n",
      "[451]\tvalidation_0-logloss:0.055972\n",
      "[452]\tvalidation_0-logloss:0.055907\n",
      "[453]\tvalidation_0-logloss:0.055872\n",
      "[454]\tvalidation_0-logloss:0.055832\n",
      "[455]\tvalidation_0-logloss:0.055762\n",
      "[456]\tvalidation_0-logloss:0.055758\n",
      "[457]\tvalidation_0-logloss:0.055632\n",
      "[458]\tvalidation_0-logloss:0.055618\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[459]\tvalidation_0-logloss:0.055525\n",
      "[460]\tvalidation_0-logloss:0.055444\n",
      "[461]\tvalidation_0-logloss:0.055394\n",
      "[462]\tvalidation_0-logloss:0.055361\n",
      "[463]\tvalidation_0-logloss:0.055354\n",
      "[464]\tvalidation_0-logloss:0.055277\n",
      "[465]\tvalidation_0-logloss:0.055262\n",
      "[466]\tvalidation_0-logloss:0.055252\n",
      "[467]\tvalidation_0-logloss:0.055259\n",
      "[468]\tvalidation_0-logloss:0.05526\n",
      "[469]\tvalidation_0-logloss:0.055261\n",
      "[470]\tvalidation_0-logloss:0.055205\n",
      "[471]\tvalidation_0-logloss:0.055172\n",
      "[472]\tvalidation_0-logloss:0.055184\n",
      "[473]\tvalidation_0-logloss:0.055186\n",
      "[474]\tvalidation_0-logloss:0.055139\n",
      "[475]\tvalidation_0-logloss:0.055155\n",
      "[476]\tvalidation_0-logloss:0.05514\n",
      "[477]\tvalidation_0-logloss:0.055122\n",
      "[478]\tvalidation_0-logloss:0.055104\n",
      "[479]\tvalidation_0-logloss:0.055072\n",
      "[480]\tvalidation_0-logloss:0.055044\n",
      "[481]\tvalidation_0-logloss:0.055001\n",
      "[482]\tvalidation_0-logloss:0.054955\n",
      "[483]\tvalidation_0-logloss:0.054924\n",
      "[484]\tvalidation_0-logloss:0.054869\n",
      "[485]\tvalidation_0-logloss:0.054867\n",
      "[486]\tvalidation_0-logloss:0.05483\n",
      "[487]\tvalidation_0-logloss:0.054757\n",
      "[488]\tvalidation_0-logloss:0.054702\n",
      "[489]\tvalidation_0-logloss:0.054703\n",
      "[490]\tvalidation_0-logloss:0.054649\n",
      "[491]\tvalidation_0-logloss:0.054628\n",
      "[492]\tvalidation_0-logloss:0.054583\n",
      "[493]\tvalidation_0-logloss:0.054549\n",
      "[494]\tvalidation_0-logloss:0.054564\n",
      "[495]\tvalidation_0-logloss:0.054583\n",
      "[496]\tvalidation_0-logloss:0.054447\n",
      "[497]\tvalidation_0-logloss:0.054469\n",
      "[498]\tvalidation_0-logloss:0.054438\n",
      "[499]\tvalidation_0-logloss:0.054464\n",
      "[500]\tvalidation_0-logloss:0.054406\n",
      "[501]\tvalidation_0-logloss:0.054369\n",
      "[502]\tvalidation_0-logloss:0.054382\n",
      "[503]\tvalidation_0-logloss:0.054366\n",
      "[504]\tvalidation_0-logloss:0.05434\n",
      "[505]\tvalidation_0-logloss:0.054299\n",
      "[506]\tvalidation_0-logloss:0.054335\n",
      "[507]\tvalidation_0-logloss:0.054328\n",
      "[508]\tvalidation_0-logloss:0.054263\n",
      "[509]\tvalidation_0-logloss:0.054239\n",
      "[510]\tvalidation_0-logloss:0.054215\n",
      "[511]\tvalidation_0-logloss:0.054202\n",
      "[512]\tvalidation_0-logloss:0.054191\n",
      "[513]\tvalidation_0-logloss:0.054147\n",
      "[514]\tvalidation_0-logloss:0.054171\n",
      "[515]\tvalidation_0-logloss:0.054107\n",
      "[516]\tvalidation_0-logloss:0.054061\n",
      "[517]\tvalidation_0-logloss:0.05408\n",
      "[518]\tvalidation_0-logloss:0.054065\n",
      "[519]\tvalidation_0-logloss:0.054014\n",
      "[520]\tvalidation_0-logloss:0.054032\n",
      "[521]\tvalidation_0-logloss:0.054019\n",
      "[522]\tvalidation_0-logloss:0.054021\n",
      "[523]\tvalidation_0-logloss:0.053973\n",
      "[524]\tvalidation_0-logloss:0.053977\n",
      "[525]\tvalidation_0-logloss:0.053948\n",
      "[526]\tvalidation_0-logloss:0.053833\n",
      "[527]\tvalidation_0-logloss:0.053838\n",
      "[528]\tvalidation_0-logloss:0.053821\n",
      "[529]\tvalidation_0-logloss:0.053801\n",
      "[530]\tvalidation_0-logloss:0.053809\n",
      "[531]\tvalidation_0-logloss:0.053766\n",
      "[532]\tvalidation_0-logloss:0.05375\n",
      "[533]\tvalidation_0-logloss:0.053754\n",
      "[534]\tvalidation_0-logloss:0.053758\n",
      "[535]\tvalidation_0-logloss:0.053755\n",
      "[536]\tvalidation_0-logloss:0.053749\n",
      "[537]\tvalidation_0-logloss:0.053689\n",
      "[538]\tvalidation_0-logloss:0.053659\n",
      "[539]\tvalidation_0-logloss:0.053666\n",
      "[540]\tvalidation_0-logloss:0.053549\n",
      "[541]\tvalidation_0-logloss:0.053551\n",
      "[542]\tvalidation_0-logloss:0.053481\n",
      "[543]\tvalidation_0-logloss:0.053502\n",
      "[544]\tvalidation_0-logloss:0.053481\n",
      "[545]\tvalidation_0-logloss:0.053406\n",
      "[546]\tvalidation_0-logloss:0.053442\n",
      "[547]\tvalidation_0-logloss:0.053442\n",
      "[548]\tvalidation_0-logloss:0.053421\n",
      "[549]\tvalidation_0-logloss:0.053402\n",
      "[550]\tvalidation_0-logloss:0.053347\n",
      "[551]\tvalidation_0-logloss:0.053312\n",
      "[552]\tvalidation_0-logloss:0.053294\n",
      "[553]\tvalidation_0-logloss:0.053308\n",
      "[554]\tvalidation_0-logloss:0.053223\n",
      "[555]\tvalidation_0-logloss:0.053175\n",
      "[556]\tvalidation_0-logloss:0.053199\n",
      "[557]\tvalidation_0-logloss:0.053169\n",
      "[558]\tvalidation_0-logloss:0.053123\n",
      "[559]\tvalidation_0-logloss:0.053144\n",
      "[560]\tvalidation_0-logloss:0.053096\n",
      "[561]\tvalidation_0-logloss:0.053134\n",
      "[562]\tvalidation_0-logloss:0.053177\n",
      "[563]\tvalidation_0-logloss:0.053097\n",
      "[564]\tvalidation_0-logloss:0.053099\n",
      "[565]\tvalidation_0-logloss:0.053048\n",
      "[566]\tvalidation_0-logloss:0.053036\n",
      "[567]\tvalidation_0-logloss:0.053024\n",
      "[568]\tvalidation_0-logloss:0.05299\n",
      "[569]\tvalidation_0-logloss:0.052932\n",
      "[570]\tvalidation_0-logloss:0.052912\n",
      "[571]\tvalidation_0-logloss:0.052932\n",
      "[572]\tvalidation_0-logloss:0.052946\n",
      "[573]\tvalidation_0-logloss:0.052928\n",
      "[574]\tvalidation_0-logloss:0.052937\n",
      "[575]\tvalidation_0-logloss:0.052934\n",
      "[576]\tvalidation_0-logloss:0.052967\n",
      "[577]\tvalidation_0-logloss:0.052971\n",
      "[578]\tvalidation_0-logloss:0.053004\n",
      "[579]\tvalidation_0-logloss:0.053021\n",
      "[580]\tvalidation_0-logloss:0.052945\n",
      "[581]\tvalidation_0-logloss:0.052974\n",
      "[582]\tvalidation_0-logloss:0.052938\n",
      "[583]\tvalidation_0-logloss:0.052965\n",
      "[584]\tvalidation_0-logloss:0.052938\n",
      "[585]\tvalidation_0-logloss:0.052918\n",
      "[586]\tvalidation_0-logloss:0.052895\n",
      "[587]\tvalidation_0-logloss:0.052914\n",
      "[588]\tvalidation_0-logloss:0.052879\n",
      "[589]\tvalidation_0-logloss:0.052868\n",
      "[590]\tvalidation_0-logloss:0.05287\n",
      "[591]\tvalidation_0-logloss:0.052901\n",
      "[592]\tvalidation_0-logloss:0.052904\n",
      "[593]\tvalidation_0-logloss:0.05289\n",
      "[594]\tvalidation_0-logloss:0.052853\n",
      "[595]\tvalidation_0-logloss:0.052859\n",
      "[596]\tvalidation_0-logloss:0.052861\n",
      "[597]\tvalidation_0-logloss:0.052817\n",
      "[598]\tvalidation_0-logloss:0.052826\n",
      "[599]\tvalidation_0-logloss:0.052811\n",
      "[600]\tvalidation_0-logloss:0.052845\n",
      "[601]\tvalidation_0-logloss:0.052835\n",
      "[602]\tvalidation_0-logloss:0.052836\n",
      "[603]\tvalidation_0-logloss:0.052772\n",
      "[604]\tvalidation_0-logloss:0.052793\n",
      "[605]\tvalidation_0-logloss:0.052779\n",
      "[606]\tvalidation_0-logloss:0.052794\n",
      "[607]\tvalidation_0-logloss:0.052762\n",
      "[608]\tvalidation_0-logloss:0.052738\n",
      "[609]\tvalidation_0-logloss:0.052731\n",
      "[610]\tvalidation_0-logloss:0.052672\n",
      "[611]\tvalidation_0-logloss:0.052667\n",
      "[612]\tvalidation_0-logloss:0.052662\n",
      "[613]\tvalidation_0-logloss:0.052616\n",
      "[614]\tvalidation_0-logloss:0.052652\n",
      "[615]\tvalidation_0-logloss:0.052661\n",
      "[616]\tvalidation_0-logloss:0.052658\n",
      "[617]\tvalidation_0-logloss:0.052642\n",
      "[618]\tvalidation_0-logloss:0.052638\n",
      "[619]\tvalidation_0-logloss:0.052555\n",
      "[620]\tvalidation_0-logloss:0.052538\n",
      "[621]\tvalidation_0-logloss:0.052526\n",
      "[622]\tvalidation_0-logloss:0.052486\n",
      "[623]\tvalidation_0-logloss:0.052416\n",
      "[624]\tvalidation_0-logloss:0.052436\n",
      "[625]\tvalidation_0-logloss:0.052398\n",
      "[626]\tvalidation_0-logloss:0.052392\n",
      "[627]\tvalidation_0-logloss:0.052395\n",
      "[628]\tvalidation_0-logloss:0.052397\n",
      "[629]\tvalidation_0-logloss:0.052409\n",
      "[630]\tvalidation_0-logloss:0.052435\n",
      "[631]\tvalidation_0-logloss:0.052394\n",
      "[632]\tvalidation_0-logloss:0.052355\n",
      "[633]\tvalidation_0-logloss:0.052344\n",
      "[634]\tvalidation_0-logloss:0.052336\n",
      "[635]\tvalidation_0-logloss:0.052305\n",
      "[636]\tvalidation_0-logloss:0.052199\n",
      "[637]\tvalidation_0-logloss:0.052182\n",
      "[638]\tvalidation_0-logloss:0.052142\n",
      "[639]\tvalidation_0-logloss:0.052124\n",
      "[640]\tvalidation_0-logloss:0.052137\n",
      "[641]\tvalidation_0-logloss:0.052084\n",
      "[642]\tvalidation_0-logloss:0.052103\n",
      "[643]\tvalidation_0-logloss:0.052099\n",
      "[644]\tvalidation_0-logloss:0.052099\n",
      "[645]\tvalidation_0-logloss:0.052116\n",
      "[646]\tvalidation_0-logloss:0.052112\n",
      "[647]\tvalidation_0-logloss:0.052088\n",
      "[648]\tvalidation_0-logloss:0.052097\n",
      "[649]\tvalidation_0-logloss:0.052055\n",
      "[650]\tvalidation_0-logloss:0.052058\n",
      "[651]\tvalidation_0-logloss:0.052078\n",
      "[652]\tvalidation_0-logloss:0.052001\n",
      "[653]\tvalidation_0-logloss:0.052\n",
      "[654]\tvalidation_0-logloss:0.051983\n",
      "[655]\tvalidation_0-logloss:0.052008\n",
      "[656]\tvalidation_0-logloss:0.052013\n",
      "[657]\tvalidation_0-logloss:0.052014\n",
      "[658]\tvalidation_0-logloss:0.052009\n",
      "[659]\tvalidation_0-logloss:0.052029\n",
      "[660]\tvalidation_0-logloss:0.052051\n",
      "[661]\tvalidation_0-logloss:0.052028\n",
      "[662]\tvalidation_0-logloss:0.052074\n",
      "[663]\tvalidation_0-logloss:0.05205\n",
      "[664]\tvalidation_0-logloss:0.052072\n",
      "[665]\tvalidation_0-logloss:0.052077\n",
      "[666]\tvalidation_0-logloss:0.05209\n",
      "[667]\tvalidation_0-logloss:0.052053\n",
      "[668]\tvalidation_0-logloss:0.052061\n",
      "[669]\tvalidation_0-logloss:0.05207\n",
      "[670]\tvalidation_0-logloss:0.052023\n",
      "[671]\tvalidation_0-logloss:0.052007\n",
      "[672]\tvalidation_0-logloss:0.052012\n",
      "[673]\tvalidation_0-logloss:0.052008\n",
      "[674]\tvalidation_0-logloss:0.051954\n",
      "[675]\tvalidation_0-logloss:0.051953\n",
      "[676]\tvalidation_0-logloss:0.051991\n",
      "[677]\tvalidation_0-logloss:0.051958\n",
      "[678]\tvalidation_0-logloss:0.05198\n",
      "[679]\tvalidation_0-logloss:0.051962\n",
      "[680]\tvalidation_0-logloss:0.051914\n",
      "[681]\tvalidation_0-logloss:0.051961\n",
      "[682]\tvalidation_0-logloss:0.05195\n",
      "[683]\tvalidation_0-logloss:0.051941\n",
      "[684]\tvalidation_0-logloss:0.051912\n",
      "[685]\tvalidation_0-logloss:0.051861\n",
      "[686]\tvalidation_0-logloss:0.051883\n",
      "[687]\tvalidation_0-logloss:0.051838\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[688]\tvalidation_0-logloss:0.051825\n",
      "[689]\tvalidation_0-logloss:0.051828\n",
      "[690]\tvalidation_0-logloss:0.051813\n",
      "[691]\tvalidation_0-logloss:0.051809\n",
      "[692]\tvalidation_0-logloss:0.051805\n",
      "[693]\tvalidation_0-logloss:0.051802\n",
      "[694]\tvalidation_0-logloss:0.051789\n",
      "[695]\tvalidation_0-logloss:0.051779\n",
      "[696]\tvalidation_0-logloss:0.051771\n",
      "[697]\tvalidation_0-logloss:0.051792\n",
      "[698]\tvalidation_0-logloss:0.051777\n",
      "[699]\tvalidation_0-logloss:0.051755\n",
      "[700]\tvalidation_0-logloss:0.051762\n",
      "[701]\tvalidation_0-logloss:0.051762\n",
      "[702]\tvalidation_0-logloss:0.051826\n",
      "[703]\tvalidation_0-logloss:0.051799\n",
      "[704]\tvalidation_0-logloss:0.05172\n",
      "[705]\tvalidation_0-logloss:0.051718\n",
      "[706]\tvalidation_0-logloss:0.051724\n",
      "[707]\tvalidation_0-logloss:0.051724\n",
      "[708]\tvalidation_0-logloss:0.051711\n",
      "[709]\tvalidation_0-logloss:0.051649\n",
      "[710]\tvalidation_0-logloss:0.051604\n",
      "[711]\tvalidation_0-logloss:0.051583\n",
      "[712]\tvalidation_0-logloss:0.051587\n",
      "[713]\tvalidation_0-logloss:0.051611\n",
      "[714]\tvalidation_0-logloss:0.051559\n",
      "[715]\tvalidation_0-logloss:0.051518\n",
      "[716]\tvalidation_0-logloss:0.051534\n",
      "[717]\tvalidation_0-logloss:0.051541\n",
      "[718]\tvalidation_0-logloss:0.051593\n",
      "[719]\tvalidation_0-logloss:0.051571\n",
      "[720]\tvalidation_0-logloss:0.051518\n",
      "[721]\tvalidation_0-logloss:0.051526\n",
      "[722]\tvalidation_0-logloss:0.051499\n",
      "[723]\tvalidation_0-logloss:0.051467\n",
      "[724]\tvalidation_0-logloss:0.051452\n",
      "[725]\tvalidation_0-logloss:0.051417\n",
      "[726]\tvalidation_0-logloss:0.051336\n",
      "[727]\tvalidation_0-logloss:0.051315\n",
      "[728]\tvalidation_0-logloss:0.051337\n",
      "[729]\tvalidation_0-logloss:0.051345\n",
      "[730]\tvalidation_0-logloss:0.051331\n",
      "[731]\tvalidation_0-logloss:0.05136\n",
      "[732]\tvalidation_0-logloss:0.051387\n",
      "[733]\tvalidation_0-logloss:0.051389\n",
      "[734]\tvalidation_0-logloss:0.051364\n",
      "[735]\tvalidation_0-logloss:0.051365\n",
      "[736]\tvalidation_0-logloss:0.051364\n",
      "[737]\tvalidation_0-logloss:0.051354\n",
      "[738]\tvalidation_0-logloss:0.051376\n",
      "[739]\tvalidation_0-logloss:0.051378\n",
      "[740]\tvalidation_0-logloss:0.05139\n",
      "[741]\tvalidation_0-logloss:0.051393\n",
      "[742]\tvalidation_0-logloss:0.051361\n",
      "[743]\tvalidation_0-logloss:0.051348\n",
      "[744]\tvalidation_0-logloss:0.051352\n",
      "[745]\tvalidation_0-logloss:0.051371\n",
      "[746]\tvalidation_0-logloss:0.051374\n",
      "[747]\tvalidation_0-logloss:0.051373\n",
      "[748]\tvalidation_0-logloss:0.05134\n",
      "[749]\tvalidation_0-logloss:0.051287\n",
      "[750]\tvalidation_0-logloss:0.051284\n",
      "[751]\tvalidation_0-logloss:0.051233\n",
      "[752]\tvalidation_0-logloss:0.051263\n",
      "[753]\tvalidation_0-logloss:0.051265\n",
      "[754]\tvalidation_0-logloss:0.051236\n",
      "[755]\tvalidation_0-logloss:0.051194\n",
      "[756]\tvalidation_0-logloss:0.051195\n",
      "[757]\tvalidation_0-logloss:0.051184\n",
      "[758]\tvalidation_0-logloss:0.051185\n",
      "[759]\tvalidation_0-logloss:0.051193\n",
      "[760]\tvalidation_0-logloss:0.051189\n",
      "[761]\tvalidation_0-logloss:0.05121\n",
      "[762]\tvalidation_0-logloss:0.051224\n",
      "[763]\tvalidation_0-logloss:0.051215\n",
      "[764]\tvalidation_0-logloss:0.051225\n",
      "[765]\tvalidation_0-logloss:0.05122\n",
      "[766]\tvalidation_0-logloss:0.051214\n",
      "[767]\tvalidation_0-logloss:0.051217\n",
      "[768]\tvalidation_0-logloss:0.051153\n",
      "[769]\tvalidation_0-logloss:0.051175\n",
      "[770]\tvalidation_0-logloss:0.051184\n",
      "[771]\tvalidation_0-logloss:0.051155\n",
      "[772]\tvalidation_0-logloss:0.051161\n",
      "[773]\tvalidation_0-logloss:0.051173\n",
      "[774]\tvalidation_0-logloss:0.051129\n",
      "[775]\tvalidation_0-logloss:0.05113\n",
      "[776]\tvalidation_0-logloss:0.051133\n",
      "[777]\tvalidation_0-logloss:0.05113\n",
      "[778]\tvalidation_0-logloss:0.051179\n",
      "[779]\tvalidation_0-logloss:0.051184\n",
      "[780]\tvalidation_0-logloss:0.05119\n",
      "[781]\tvalidation_0-logloss:0.051223\n",
      "[782]\tvalidation_0-logloss:0.05124\n",
      "[783]\tvalidation_0-logloss:0.051257\n",
      "[784]\tvalidation_0-logloss:0.051237\n",
      "[785]\tvalidation_0-logloss:0.051253\n",
      "[786]\tvalidation_0-logloss:0.051251\n",
      "[787]\tvalidation_0-logloss:0.05127\n",
      "[788]\tvalidation_0-logloss:0.051229\n",
      "[789]\tvalidation_0-logloss:0.051234\n",
      "[790]\tvalidation_0-logloss:0.051248\n",
      "[791]\tvalidation_0-logloss:0.051239\n",
      "[792]\tvalidation_0-logloss:0.051252\n",
      "[793]\tvalidation_0-logloss:0.051244\n",
      "[794]\tvalidation_0-logloss:0.051216\n",
      "[795]\tvalidation_0-logloss:0.051157\n",
      "[796]\tvalidation_0-logloss:0.051146\n",
      "[797]\tvalidation_0-logloss:0.051136\n",
      "[798]\tvalidation_0-logloss:0.051145\n",
      "[799]\tvalidation_0-logloss:0.051095\n",
      "[800]\tvalidation_0-logloss:0.051038\n",
      "[801]\tvalidation_0-logloss:0.051042\n",
      "[802]\tvalidation_0-logloss:0.051068\n",
      "[803]\tvalidation_0-logloss:0.051057\n",
      "[804]\tvalidation_0-logloss:0.051037\n",
      "[805]\tvalidation_0-logloss:0.051038\n",
      "[806]\tvalidation_0-logloss:0.051051\n",
      "[807]\tvalidation_0-logloss:0.051066\n",
      "[808]\tvalidation_0-logloss:0.051082\n",
      "[809]\tvalidation_0-logloss:0.05111\n",
      "[810]\tvalidation_0-logloss:0.051078\n",
      "[811]\tvalidation_0-logloss:0.051088\n",
      "[812]\tvalidation_0-logloss:0.05107\n",
      "[813]\tvalidation_0-logloss:0.05109\n",
      "[814]\tvalidation_0-logloss:0.051096\n",
      "[815]\tvalidation_0-logloss:0.051063\n",
      "[816]\tvalidation_0-logloss:0.051078\n",
      "[817]\tvalidation_0-logloss:0.05112\n",
      "[818]\tvalidation_0-logloss:0.051098\n",
      "[819]\tvalidation_0-logloss:0.051084\n",
      "[820]\tvalidation_0-logloss:0.051089\n",
      "[821]\tvalidation_0-logloss:0.051091\n",
      "[822]\tvalidation_0-logloss:0.051126\n",
      "[823]\tvalidation_0-logloss:0.051129\n",
      "[824]\tvalidation_0-logloss:0.051108\n",
      "[825]\tvalidation_0-logloss:0.051081\n",
      "[826]\tvalidation_0-logloss:0.051024\n",
      "[827]\tvalidation_0-logloss:0.051035\n",
      "[828]\tvalidation_0-logloss:0.051073\n",
      "[829]\tvalidation_0-logloss:0.051068\n",
      "[830]\tvalidation_0-logloss:0.051069\n",
      "[831]\tvalidation_0-logloss:0.05107\n",
      "[832]\tvalidation_0-logloss:0.051078\n",
      "[833]\tvalidation_0-logloss:0.051109\n",
      "[834]\tvalidation_0-logloss:0.05114\n",
      "[835]\tvalidation_0-logloss:0.051139\n",
      "[836]\tvalidation_0-logloss:0.051149\n",
      "[837]\tvalidation_0-logloss:0.051157\n",
      "[838]\tvalidation_0-logloss:0.051101\n",
      "[839]\tvalidation_0-logloss:0.051059\n",
      "[840]\tvalidation_0-logloss:0.051035\n",
      "[841]\tvalidation_0-logloss:0.051051\n",
      "[842]\tvalidation_0-logloss:0.051019\n",
      "[843]\tvalidation_0-logloss:0.051027\n",
      "[844]\tvalidation_0-logloss:0.051023\n",
      "[845]\tvalidation_0-logloss:0.051014\n",
      "[846]\tvalidation_0-logloss:0.050978\n",
      "[847]\tvalidation_0-logloss:0.050986\n",
      "[848]\tvalidation_0-logloss:0.051006\n",
      "[849]\tvalidation_0-logloss:0.051006\n",
      "[850]\tvalidation_0-logloss:0.050975\n",
      "[851]\tvalidation_0-logloss:0.050924\n",
      "[852]\tvalidation_0-logloss:0.050981\n",
      "[853]\tvalidation_0-logloss:0.050986\n",
      "[854]\tvalidation_0-logloss:0.050999\n",
      "[855]\tvalidation_0-logloss:0.051015\n",
      "[856]\tvalidation_0-logloss:0.051002\n",
      "[857]\tvalidation_0-logloss:0.051028\n",
      "[858]\tvalidation_0-logloss:0.051043\n",
      "[859]\tvalidation_0-logloss:0.051069\n",
      "[860]\tvalidation_0-logloss:0.05104\n",
      "[861]\tvalidation_0-logloss:0.051037\n",
      "[862]\tvalidation_0-logloss:0.051034\n",
      "[863]\tvalidation_0-logloss:0.05105\n",
      "[864]\tvalidation_0-logloss:0.051067\n",
      "[865]\tvalidation_0-logloss:0.051106\n",
      "[866]\tvalidation_0-logloss:0.051126\n",
      "[867]\tvalidation_0-logloss:0.051137\n",
      "[868]\tvalidation_0-logloss:0.051116\n",
      "[869]\tvalidation_0-logloss:0.051134\n",
      "[870]\tvalidation_0-logloss:0.051117\n",
      "[871]\tvalidation_0-logloss:0.051115\n",
      "[872]\tvalidation_0-logloss:0.051127\n",
      "[873]\tvalidation_0-logloss:0.05112\n",
      "[874]\tvalidation_0-logloss:0.051123\n",
      "[875]\tvalidation_0-logloss:0.051123\n",
      "[876]\tvalidation_0-logloss:0.05116\n",
      "[877]\tvalidation_0-logloss:0.051147\n",
      "[878]\tvalidation_0-logloss:0.051142\n",
      "[879]\tvalidation_0-logloss:0.051151\n",
      "[880]\tvalidation_0-logloss:0.051158\n",
      "[881]\tvalidation_0-logloss:0.051153\n",
      "[882]\tvalidation_0-logloss:0.051168\n",
      "[883]\tvalidation_0-logloss:0.051183\n",
      "[884]\tvalidation_0-logloss:0.051166\n",
      "[885]\tvalidation_0-logloss:0.05115\n",
      "[886]\tvalidation_0-logloss:0.05119\n",
      "[887]\tvalidation_0-logloss:0.05113\n",
      "[888]\tvalidation_0-logloss:0.051128\n",
      "[889]\tvalidation_0-logloss:0.051138\n",
      "[890]\tvalidation_0-logloss:0.051185\n",
      "[891]\tvalidation_0-logloss:0.051177\n",
      "[892]\tvalidation_0-logloss:0.051123\n",
      "[893]\tvalidation_0-logloss:0.051136\n",
      "[894]\tvalidation_0-logloss:0.051123\n",
      "[895]\tvalidation_0-logloss:0.051081\n",
      "[896]\tvalidation_0-logloss:0.051078\n",
      "[897]\tvalidation_0-logloss:0.051074\n",
      "[898]\tvalidation_0-logloss:0.05104\n",
      "[899]\tvalidation_0-logloss:0.051036\n",
      "[900]\tvalidation_0-logloss:0.051008\n",
      "[901]\tvalidation_0-logloss:0.051029\n",
      "[902]\tvalidation_0-logloss:0.05099\n",
      "[903]\tvalidation_0-logloss:0.050992\n",
      "[904]\tvalidation_0-logloss:0.050986\n",
      "[905]\tvalidation_0-logloss:0.050987\n",
      "[906]\tvalidation_0-logloss:0.05101\n",
      "[907]\tvalidation_0-logloss:0.05102\n",
      "[908]\tvalidation_0-logloss:0.051002\n",
      "[909]\tvalidation_0-logloss:0.051003\n",
      "[910]\tvalidation_0-logloss:0.051029\n",
      "[911]\tvalidation_0-logloss:0.05104\n",
      "[912]\tvalidation_0-logloss:0.051014\n",
      "[913]\tvalidation_0-logloss:0.050983\n",
      "[914]\tvalidation_0-logloss:0.050989\n",
      "[915]\tvalidation_0-logloss:0.050972\n",
      "[916]\tvalidation_0-logloss:0.050984\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[917]\tvalidation_0-logloss:0.050993\n",
      "[918]\tvalidation_0-logloss:0.050991\n",
      "[919]\tvalidation_0-logloss:0.050984\n",
      "[920]\tvalidation_0-logloss:0.051026\n",
      "[921]\tvalidation_0-logloss:0.051019\n",
      "[922]\tvalidation_0-logloss:0.051032\n",
      "[923]\tvalidation_0-logloss:0.051023\n",
      "[924]\tvalidation_0-logloss:0.051015\n",
      "[925]\tvalidation_0-logloss:0.050967\n",
      "[926]\tvalidation_0-logloss:0.050971\n",
      "[927]\tvalidation_0-logloss:0.050952\n",
      "[928]\tvalidation_0-logloss:0.050912\n",
      "[929]\tvalidation_0-logloss:0.050893\n",
      "[930]\tvalidation_0-logloss:0.050879\n",
      "[931]\tvalidation_0-logloss:0.050896\n",
      "[932]\tvalidation_0-logloss:0.050879\n",
      "[933]\tvalidation_0-logloss:0.05084\n",
      "[934]\tvalidation_0-logloss:0.050841\n",
      "[935]\tvalidation_0-logloss:0.050831\n",
      "[936]\tvalidation_0-logloss:0.050808\n",
      "[937]\tvalidation_0-logloss:0.05077\n",
      "[938]\tvalidation_0-logloss:0.050803\n",
      "[939]\tvalidation_0-logloss:0.050806\n",
      "[940]\tvalidation_0-logloss:0.050853\n",
      "[941]\tvalidation_0-logloss:0.050862\n",
      "[942]\tvalidation_0-logloss:0.050849\n",
      "[943]\tvalidation_0-logloss:0.050879\n",
      "[944]\tvalidation_0-logloss:0.050849\n",
      "[945]\tvalidation_0-logloss:0.05086\n",
      "[946]\tvalidation_0-logloss:0.050848\n",
      "[947]\tvalidation_0-logloss:0.050843\n",
      "[948]\tvalidation_0-logloss:0.050865\n",
      "[949]\tvalidation_0-logloss:0.050871\n",
      "[950]\tvalidation_0-logloss:0.050914\n",
      "[951]\tvalidation_0-logloss:0.050821\n",
      "[952]\tvalidation_0-logloss:0.050855\n",
      "[953]\tvalidation_0-logloss:0.050885\n",
      "[954]\tvalidation_0-logloss:0.050867\n",
      "[955]\tvalidation_0-logloss:0.050838\n",
      "[956]\tvalidation_0-logloss:0.050817\n",
      "[957]\tvalidation_0-logloss:0.050813\n",
      "[958]\tvalidation_0-logloss:0.050788\n",
      "[959]\tvalidation_0-logloss:0.050803\n",
      "[960]\tvalidation_0-logloss:0.050832\n",
      "[961]\tvalidation_0-logloss:0.050808\n",
      "[962]\tvalidation_0-logloss:0.050808\n",
      "[963]\tvalidation_0-logloss:0.05082\n",
      "[964]\tvalidation_0-logloss:0.050837\n",
      "[965]\tvalidation_0-logloss:0.050838\n",
      "[966]\tvalidation_0-logloss:0.050867\n",
      "[967]\tvalidation_0-logloss:0.05089\n",
      "[968]\tvalidation_0-logloss:0.050868\n",
      "[969]\tvalidation_0-logloss:0.050897\n",
      "[970]\tvalidation_0-logloss:0.050888\n",
      "[971]\tvalidation_0-logloss:0.050896\n",
      "[972]\tvalidation_0-logloss:0.050846\n",
      "[973]\tvalidation_0-logloss:0.050849\n",
      "[974]\tvalidation_0-logloss:0.050864\n",
      "[975]\tvalidation_0-logloss:0.05087\n",
      "[976]\tvalidation_0-logloss:0.05087\n",
      "[977]\tvalidation_0-logloss:0.050834\n",
      "[978]\tvalidation_0-logloss:0.05086\n",
      "[979]\tvalidation_0-logloss:0.050873\n",
      "[980]\tvalidation_0-logloss:0.050919\n",
      "[981]\tvalidation_0-logloss:0.050921\n",
      "[982]\tvalidation_0-logloss:0.050916\n",
      "[983]\tvalidation_0-logloss:0.050923\n",
      "[984]\tvalidation_0-logloss:0.050928\n",
      "[985]\tvalidation_0-logloss:0.050924\n",
      "[986]\tvalidation_0-logloss:0.050926\n",
      "[987]\tvalidation_0-logloss:0.050944\n",
      "[988]\tvalidation_0-logloss:0.050922\n",
      "[989]\tvalidation_0-logloss:0.05094\n",
      "[990]\tvalidation_0-logloss:0.050944\n",
      "[991]\tvalidation_0-logloss:0.050953\n",
      "[992]\tvalidation_0-logloss:0.051029\n",
      "[993]\tvalidation_0-logloss:0.051029\n",
      "[994]\tvalidation_0-logloss:0.051012\n",
      "[995]\tvalidation_0-logloss:0.051004\n",
      "[996]\tvalidation_0-logloss:0.051006\n",
      "[997]\tvalidation_0-logloss:0.051032\n",
      "[998]\tvalidation_0-logloss:0.051042\n",
      "[999]\tvalidation_0-logloss:0.051023\n",
      "[1000]\tvalidation_0-logloss:0.051003\n",
      "[1001]\tvalidation_0-logloss:0.050997\n",
      "[1002]\tvalidation_0-logloss:0.050996\n",
      "[1003]\tvalidation_0-logloss:0.050995\n",
      "[1004]\tvalidation_0-logloss:0.051002\n",
      "[1005]\tvalidation_0-logloss:0.050969\n",
      "[1006]\tvalidation_0-logloss:0.050973\n",
      "[1007]\tvalidation_0-logloss:0.050965\n",
      "[1008]\tvalidation_0-logloss:0.050963\n",
      "[1009]\tvalidation_0-logloss:0.050916\n",
      "[1010]\tvalidation_0-logloss:0.050933\n",
      "[1011]\tvalidation_0-logloss:0.050879\n",
      "[1012]\tvalidation_0-logloss:0.050873\n",
      "[1013]\tvalidation_0-logloss:0.050844\n",
      "[1014]\tvalidation_0-logloss:0.050832\n",
      "[1015]\tvalidation_0-logloss:0.050857\n",
      "[1016]\tvalidation_0-logloss:0.050871\n",
      "[1017]\tvalidation_0-logloss:0.050874\n",
      "[1018]\tvalidation_0-logloss:0.05086\n",
      "[1019]\tvalidation_0-logloss:0.050907\n",
      "[1020]\tvalidation_0-logloss:0.050902\n",
      "[1021]\tvalidation_0-logloss:0.050864\n",
      "[1022]\tvalidation_0-logloss:0.050864\n",
      "[1023]\tvalidation_0-logloss:0.050817\n",
      "[1024]\tvalidation_0-logloss:0.050843\n",
      "[1025]\tvalidation_0-logloss:0.050874\n",
      "[1026]\tvalidation_0-logloss:0.050866\n",
      "[1027]\tvalidation_0-logloss:0.050883\n",
      "[1028]\tvalidation_0-logloss:0.050906\n",
      "[1029]\tvalidation_0-logloss:0.050922\n",
      "[1030]\tvalidation_0-logloss:0.05092\n",
      "[1031]\tvalidation_0-logloss:0.05093\n",
      "[1032]\tvalidation_0-logloss:0.050953\n",
      "[1033]\tvalidation_0-logloss:0.050957\n",
      "[1034]\tvalidation_0-logloss:0.050978\n",
      "[1035]\tvalidation_0-logloss:0.050978\n",
      "[1036]\tvalidation_0-logloss:0.050981\n",
      "[1037]\tvalidation_0-logloss:0.050973\n",
      "Stopping. Best iteration:\n",
      "[937]\tvalidation_0-logloss:0.05077\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.5, colsample_bylevel=1, colsample_bytree=1,\n",
       "       gamma=0, learning_rate=0.1, max_delta_step=0, max_depth=8,\n",
       "       min_child_weight=1, missing=None, n_estimators=2000, nthread=-1,\n",
       "       objective='binary:logistic', reg_alpha=0, reg_lambda=1,\n",
       "       scale_pos_weight=1, seed=0, silent=True, subsample=1)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = XGBClassifier(\n",
    " learning_rate =0.1,\n",
    " n_estimators=2000,\n",
    " max_depth=8,\n",
    ")\n",
    "\n",
    "#model.fit(X, y, eval_set=[(X, y), (x_test, y_test)], eval_metric='logloss', verbose=True)\n",
    "model.fit(x_train, y_train, eval_set=[(x_test, y_test)], eval_metric='logloss',early_stopping_rounds=500, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "pickle.dump(model, open(\"xgb.model.dat\", \"wb\"))\n",
    "#loaded_model = pickle.load(open(\"pima.pickle.dat\", \"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = pd.read_csv('data/test.csv', header=None, names = list(range(0,max_features)))\n",
    "\n",
    "number_of_nan = test.isnull().sum(axis=1)\n",
    "test = test.fillna(0)\n",
    "test = test.astype(int)\n",
    "number_valid = max_features - number_of_nan\n",
    "test_describe = test.apply(pd.DataFrame.describe, axis=1)\n",
    "test = pd.concat([test, number_of_nan, number_valid, test_describe], axis=1)\n",
    "test.columns.values[1024] = \"number_of_nan\"\n",
    "test.columns.values[1025] = \"number_valid\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_iter = model.best_iteration\n",
    "y_pred = model.predict_proba(test, ntree_limit = best_iter)[:, 1]\n",
    "\n",
    "df = pd.DataFrame(y_pred)\n",
    "df.to_csv(\"data/sgb_prob.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9561"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_iter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
